{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TorchServe 安装与运行\n",
    "- 基于 Python 和 java8\n",
    "```shell\n",
    "haha7@carpediem:~$ sudo apt-get install openjdk-11-jdk\n",
    "```\n",
    "- 创建 conda 虚拟环境\n",
    "\n",
    "```shell\n",
    "# 创建虚拟环境\n",
    "haha7@carpediem:~$ conda create -n torchserve\n",
    "haha7@carpediem:~$ conda activate torchserve\n",
    "\n",
    "# 安装pytorch包\n",
    "(torchserve) haha7@carpediem:~$ conda install -c pytorch -c powerai pytorch torchtext torchvision\n",
    "\n",
    "# 安装torchserve\n",
    "(torchserve) haha7@carpediem:~$ conda install -c pytorch torchserve torch-model-archiver\n",
    "```\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "- 安装，利用 docker 运行\n",
    "```shell\n",
    "haha7@carpediem:~$ docker run --rm -it -p 8080:8080 -p 8081:8081 pytorch/torchserve:latest\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 拉取并配置docker容器\n",
    "```shell\n",
    "# 1. 拉取 torchserve 容器\n",
    "hahaha@carpediem:~$ docker pull pytorch/torchserve\n",
    "\n",
    "# 2. 进入容器命令行界面\n",
    "hahaha@carpediem:~$ docker run -it pytorch/torchserve /bin/bash\n",
    "\n",
    "# 3. 安装必要的包\n",
    "model-server@0ab89874a541:~$ pip install -f https://download.pytorch.org/whl/torch_stable.html torchserve torch-model-archiver\n",
    "\n",
    "# 4. 将 torch-model-archiver 脚本路径添加到 PATH 中         \n",
    "model-server@0ab89874a541:~$ vim ~/.bashrc\n",
    "# 在文件最后一行添加：export PATH=/home/model-server/.local/bin:$PATH\n",
    "\n",
    "# 5. 然后激活设置，可以运行命令 torch-model-archiver\n",
    "model-server@0ab89874a541:~$ source ~/.bashrc\n",
    "model-server@0ab89874a541:~$ torch-model-archiver\n",
    "usage: torch-model-archiver [-h] --model-name MODEL_NAME --serialized-file\n",
    "......\n",
    "\n",
    "# 6. 查看容器内内容\n",
    "model-server@0ab89874a541:~$ ls -al\n",
    "total 52\n",
    "drwxr-xr-x 1 model-server model-server 4096 May 24 09:49 .\n",
    "drwxr-xr-x 1 root         root         4096 Apr 11 21:04 ..\n",
    "-rw-r--r-- 1 model-server model-server  220 Apr  4  2018 .bash_logout\n",
    "-rw-r--r-- 1 model-server model-server 3856 May 24 09:49 .bashrc\n",
    "drwxr-xr-x 3 model-server model-server 4096 May 24 09:38 .cache\n",
    "drwx------ 4 model-server model-server 4096 May 24 09:42 .local\n",
    "-rw-r--r-- 1 model-server model-server  807 Apr  4  2018 .profile\n",
    "-rw------- 1 model-server model-server  808 May 24 09:49 .viminfo\n",
    "-rw-rw-r-- 1 root         root          166 Apr 10 22:51 config.properties\n",
    "drwxr-xr-x 2 model-server root         4096 Apr 11 21:04 model-store\n",
    "drwxr-xr-x 1 model-server root         4096 May 24 09:42 tmp\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 将模型添加到容器内\n",
    "\n",
    "```shell\n",
    "# 1. 查看上一步运行的容器的ID, 活跃的容器才能向里面添加模型\n",
    "yangbin7@carpediem:~$ docker ps -a\n",
    "CONTAINER ID        IMAGE                COMMAND                  CREATED             \n",
    "11504561baf9  \n",
    "\n",
    "# 2.　下载模型 `.pth` ，然后将文件上传到容器内的`model-store`文件夹，需要指明容器的ID\n",
    "hahaha@carpediem:~$ docker cp densenet161-8d451a50.pth 11504561baf9:/home/model-server/model-store\n",
    "\n",
    "```\n",
    "\n",
    "# 创建模型脚本 `model.py`,并上传到容器内文件夹`app`\n",
    "\n",
    "```python\n",
    "from torchvision.models.densenet import DenseNet\n",
    "\n",
    "class ImageClassifier(DenseNet):\n",
    "    def __init__(self):\n",
    "        super(ImageClassifier, self).__init__(48, (6, 12, 36, 24), 96)\n",
    "    \n",
    "    def load_state_dict(self, state_dict, strict=True):\n",
    "        # '.'s are no longer allowed in module names, but previous _DenseLayer\n",
    "        # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'.\n",
    "        # They are also in the checkpoints in model_urls. This pattern is used\n",
    "        # to find such keys.\n",
    "        # Credit - https://github.com/pytorch/vision/blob/master/torchvision/models/densenet.py\n",
    "        # #def _load_state_dict()\n",
    "        import re\n",
    "        pattern = re.compile(\n",
    "            r'^(.*denselayer\\d+\\.(?:norm|relu|conv))\\.((?:[12])\\.('\n",
    "            r'?:weight|bias|running_mean|running_var))$')\n",
    "        \n",
    "        for key in list(state_dict.keys()):\n",
    "            res = pattern.match(key)\n",
    "            if res:\n",
    "                new_key = res.group(1) + res.group(2)\n",
    "                state_dict[new_key] = state_dict[key]\n",
    "                del state_dict[key]\n",
    "        \n",
    "        return super(ImageClassifier, self).load_state_dict(state_dict, strict)\n",
    "```        \n",
    "上传文件:\n",
    "```shell\n",
    "hahaha@carpediem:~$ docker cp model.py 11504561baf9:/home/model-server/app\n",
    "```\n",
    "额外的文件,保存类别名:\n",
    "```shell\n",
    "hahaha@carpediem:~$ docker cp index_to_name.json 11504561baf9:/home/model-server/app\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 配置应用\n",
    "```shell\n",
    "# 1. 上一步运行的容器中,配置应用\n",
    "model-server@11504561baf9:~$ torch-model-archiver --model-name densenet161 \\\n",
    "--version 1.0 \\\n",
    "--model-file app/model.py \\\n",
    "--serialized-file model-store/densenet161-8d451a50.pth \\\n",
    "--handler image_classifier \\\n",
    "--extra-files app/index_to_name.json\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 三种API\n",
    "- TorchServe推理APIs的描述信息：\n",
    "```shell\n",
    "haha7@carpediem:~$ curl -X OPTIONS http://localhost:8080\n",
    "{\n",
    "  \"code\": 405,\n",
    "  \"type\": \"MethodNotAllowedException\",\n",
    "  \"message\": \"Requested method is not allowed, please refer to API document.\"\n",
    "}\n",
    "```      \n",
    "- TorchServe健康检查：  \n",
    "```shell\n",
    "haha7@carpediem:~$ curl http://localhost:8080/ping\n",
    "{\n",
    "  \"status\": \"Healthy\"\n",
    "}\n",
    "``` \n",
    "- 模型预测API    \n",
    "                                   \n",
    "     - 利用加载的模型的默认版本，调用 `POST /predictions/{model_name}`\n",
    "```shell\n",
    "haha7@carpediem:~$ curl -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg\n",
    "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
    "                                 Dload  Upload   Total   Spent    Left  Speed\n",
    "100  108k  100  108k    0     0   6127      0  0:00:18  0:00:18 --:--:-- 11575\n",
    "haha7@carpediem:~$ curl -X POST http://localhost:8080/predictions/resnet-18 -T kitten.jpg\n",
    "{\n",
    "  \"code\": 404,\n",
    "  \"type\": \"ModelNotFoundException\",\n",
    "  \"message\": \"Model not found: resnet-18\"\n",
    "}\n",
    "```\n",
    "\n",
    " - 利用特定版本的模型,`POST /predictions/{model_name}/{version}`\n",
    "```shell\n",
    " haha7@carpediem:~$ curl -X POST http://localhost:8080/predictions/resnet-18/2.0 -F \"data=@kitten.jpg\"\n",
    "```\n",
    "返回结果：\n",
    "```json\n",
    "{\n",
    "    \"class\": \"n02123045 tabby, tabby cat\",\n",
    "    \"probability\": 0.42514491081237793\n",
    "}\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 预置的模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 图像分类器\n",
    "https://github.com/pytorch/serve/tree/master/examples/image_classifier   \n",
    "输入：RGB图像   \n",
    "输出：概率最高的5类及对应的概率\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 将 `.pth` 模型下载到文件夹 `model_store`\n",
    "- 在文件夹`app` 创建脚本 `model.py`，和分类的类别与类名的对应关系文件`index_to_name.json`\n",
    "\n",
    "```python\n",
    "from torchvision.models.densenet import DenseNet\n",
    "\n",
    "class ImageClassifier(DenseNet):\n",
    "    def __init__(self):\n",
    "        super(ImageClassifier, self).__init__(48, (6, 12, 36, 24), 96)\n",
    "\n",
    "    def load_state_dict(self, state_dict, strict=True):\n",
    "        # '.'s are no longer allowed in module names, but previous _DenseLayer\n",
    "        # has keys 'norm.1', 'relu.1', 'conv.1', 'norm.2', 'relu.2', 'conv.2'.\n",
    "        # They are also in the checkpoints in model_urls. This pattern is used\n",
    "        # to find such keys.\n",
    "        # Credit - https://github.com/pytorch/vision/blob/master/torchvision/models/densenet.py#def _load_state_dict()\n",
    "        import re\n",
    "        pattern = re.compile(r'^(.*denselayer\\d+\\.(?:norm|relu|conv))\\.((?:[12])\\.(?:weight|bias|running_mean|running_var))$')\n",
    "\n",
    "        for key in list(state_dict.keys()):\n",
    "            res = pattern.match(key)\n",
    "            if res:\n",
    "                new_key = res.group(1) + res.group(2)\n",
    "                state_dict[new_key] = state_dict[key]\n",
    "                del state_dict[key]\n",
    "\n",
    "        return super(ImageClassifier, self).load_state_dict(state_dict, strict)\n",
    "```\n",
    "- 将模型和脚本注册到TorchServe\n",
    "```bash\n",
    "haha7@carpediem:~$ docker run -it pytorch/torchserve /bin/bash\n",
    "model-server@54121b3750a7:~$ \n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
